#!/usr/bin/perl

use strict;
use warnings;
use Data::Dumper;
use Algorithm::NaiveBayes;
use File::Util;
use  Term::ProgressBar;
use Lingua::StopWords qw( getStopWords );

$|++;

my $appli_name = shift;
my $corpus = shift;
my $output_file = shift;

# Naives Bayes Object
my $nb = Algorithm::NaiveBayes->new;

#Misc Var
my $fh;
my $exception="";
my %count=();
my @corpus_list=();
my $total_lines=0;
my $stopWords=undef;

#Loading previous state
if ( -f "$output_file") {
    print "> Corpus exists, loading it...\n";
    $nb->restore_state($output_file);    
}

# Loading Stop Word list
$stopWords=getStopWords('en');
for my $elt( keys %{getStopWords('fr')}) { $$stopWords{$elt}++;}

# Building corpus file list
if (-f "$corpus" ) {
	print "> Corpus is a file : $corpus";
	push @corpus_list, $corpus;
	$total_lines= File::Util->new->line_count($corpus);
}

if (-d "$corpus" ) {
	print "> Corpus is a directory : $corpus\n";
	for my $f ( File::Util->new->list_dir( $corpus, qw/--pattern=\.log$ --files-only --with-paths/ )) {
    		print "\t> Adding \'$f\' in corpus list\n";
		$total_lines+= File::Util->new->line_count($f);
		push @corpus_list, $f;
	}
}

print "> Nb File to analyse :", scalar @corpus_list, " with $total_lines lines.\n";

my $progress = Term::ProgressBar->new({count => $total_lines, ETA => 'linear'});

for my $f ( @corpus_list ){
    print "\n> Training IA with : $f\n"; 
    my $label=$f;
    $label =~ s/.+\/(.+?)\.log$/$1/;
    my @labels =( "$appli_name" );
    push @labels, split "__", $label;
    print "> Training IA for labels : [ ", join(", ", @labels), " ]\n"; 

    
    open ( $fh, '<', $f);
    while (<$fh>) {
	if ( /;$appli_name;/ ) {
	    if ( $exception !~ /^\s*$/ ) {
		$exception =~ s/\n/ /g;
		$exception =~ s/;/ /g;
		$exception =~ s/\)/ /g;
		$exception =~ s/\(/ /g;
		$exception =~ s/:/ /g;
		$exception =~ s/=/ /g;
		$exception =~ s/,/ /g;
		$exception =~ s/\./ /g;
		$exception =~ s/\n/ /g;
		$exception =~ s/</ /g;
		$exception =~ s/>/ /g;
		$exception =~ s/\// /g;
		$exception =~ s/\s+/ /g;
		my @tokens=split / /, $exception;

		foreach my $word ( grep { !$stopWords->{$_} } @tokens) {
		    $count{$word}++;
		}

		$nb-> add_instance (
			attributes => {%count}, 
			label =>  [ $appli_name, $label ]
			);
		%count=();
	    }
	    $exception=$_;
	    $progress->update();
	} else {
	    $exception.=$_;
	    $progress->update();
	}

    }
    close($fh);
}
$nb->train;

$nb->save_state("$output_file");
exit(0);
